﻿#!/usr/bin/python
# -*- coding: utf-8 -*-

import string
import shutil
import glob
import re
import sys

# removing dependencies from pepe library in this case
# because this is good demonstration file for python beginners
# they are to be able to run it as standalone script 

#import pepe.p_txt.bigtxt as bt

n = '\x0D\x0A'

one_by_one = True

class lines:
    ''' read big text file as iterator
    '''
    def __init__(self , flpath_s):
        self.flpath_s = flpath_s
        self.sfile_fl = open(flpath_s, 'rb') 
    def __getitem__( self , index ):
        line_s = self.sfile_fl.readline()
        if line_s:
            return line_s
        else:
            raise IndexError

def findcoldelimiter_line(line):
    ''' guess what text file column delimiter is used 

    if cd == '': cd = findcoldelimiter_line(line)

    >>> findcoldelimiter_line("a|^b;^c~^d~")
    '^'
    '''
    c = 0; r = '' 
    for cs in [',', '\t', ';', '^', '~', '|']:
        if string.count(line, cs) > c:
            c = string.count(line, cs); r = cs
    return r

def report_issue(lines_list, br, bw):
    '''
    '''
    print "count of bytes read: ", str(br)
    print "count of bytes written: ", str(bw)
    for l in lines_list:
        print l 

def main(filesmask='DENNMXF 57130 rows Txt Qual UTF-8_txt_dat_modify.#'):
    """ Repair rows by counting total number of column delimiters present.

    Pseudo code (lines folded more than once):
    # line by line
       # add line into line_buffer
          # is header:
              # store cdelimiters count as reference R
    O+        # write line_buffer to output
              # line_buffer = ''
          # not header:
    O+        # LDC (line delimiters count) == R
                  # write line_buffer to output
                  # line_buffer = ''
              # LDC > R
    D+            # write to dropped file
                  # line_buffer = ''
              # LDC < R
    E+            # write to errors log
    """
    for inpfile in glob.glob(filesmask):
    
        fname = string.split(inpfile, '.')[0]
    
        # 2010-10-22_2257 vp
        ## using underscore instead dot because .# utilities 
        ## cut out everything after dot and then is not clear 
        ## whether utility was run on original file or on output
        ## of other .# utility that processed given file
        o = open(fname + '_repair_row_output.#', 'wb')
        e = open(fname + '_repair_row_errors.#', 'wb')
        d = open(fname + '_repair_row_dropped.#', 'wb')
    
        line_buffer = ''
    
        lino = 0
        cnt_errors = 1

        cnt_above_reference = 0
        cnt_below_reference = 0
        cnt_equal_reference = 0
        
        cnt_joined_line = 0
 
        cnt_bytes_read = 0
        cnt_bytes_written = 0
        ref_cols_cnt = 0
    
        cd = ''
        header_line = ''
        sample_line = ''
        
        lines_list = []
    
        #for line in bt.lines(inpfile):
        for line in lines(inpfile):
            if cd == '': cd = findcoldelimiter_line(line)
            lino += 1
            
            line = re.sub('\x0D\x0A', '', line)
            # remove single unix/mac end of lines, vp 2010-10-13_1756
            line = re.sub('\x0D', '', line)
            line = re.sub('\x0A', '', line)
            
            # keep lines buffer to debug problem lines
            if lino < 5:
                lines_list.append(line)
            else:
                lines_list.append(line)
                lines_list.pop(0)
            
            cnt_bytes_read += len(line)
            
            # remove tabs in pipe delimited text file
            if cd <> '\t': line = re.sub('\t', ' ', line)

            # add line to line_buffer
            if line_buffer == '':
                line_buffer = line[:]
            else:
                line_buffer = line_buffer + ' ' + line[:]
                cnt_joined_line += 1
                # correction for space added
                cnt_bytes_written -= 1

            cols = string.split(line_buffer, cd)

            if ref_cols_cnt == 0: 
                ref_cols_cnt = len(cols)
                header_line = 'HEADER_LINE:' + cd + line_buffer
            if lino == 2: 
                sample_line = 'SAMPLE_LINE:' + cd + line_buffer

            # controlsums checking on original line
            cols_original_line = string.split(line_buffer, cd)
            if len(cols_original_line) == ref_cols_cnt: cnt_equal_reference += 1
            if len(cols_original_line) >  ref_cols_cnt: cnt_above_reference += 1
            if len(cols_original_line) <  ref_cols_cnt: cnt_below_reference += 1

            if len(cols) in (ref_cols_cnt, ):
                # number of column delimiters equals expected value 
                # (split on or around delimiter position)
                o.write(line_buffer + '\x0D\x0A')
                cnt_bytes_written += len(line_buffer)
                # report if read and written bytes differ
                if cnt_bytes_read <> cnt_bytes_written: 
                    report_issue(lines_list, br=cnt_bytes_read, bw=cnt_bytes_written)
                    if one_by_one: sys.exit(2)
                
                line_buffer = ''
                
            elif len(cols) > ref_cols_cnt:
                # number of column delimiters exceeds expected value 
                # (additional cdelimiters in text)
                d.write(n + header_line + n)
                d.write(sample_line + n)
                d.write('PROBLEM LINE NO: ' + str(lino) + cd + line_buffer + '\x0D\x0A')

                if 1: # custom repair of delimiters
                    #if re.search('^71022009', line_buffer):
                    # it is safer to catch lines to repair via lino ... if not too many with same problem
                    if lino == 10324:
                        '''
                        PROBLEM LINE NO: 10324|1|SKANDIA F|RS{KRINGS AB..

                        1|SKANDIA F|RS{KRINGS AB|..
                        1|SKANDIA FORSAKRINGS AB|..
                       
                        'SKANDIA F\|RS\{KRINGS AB', 'SKANDIA FORSAKRINGS AB'
                        '''
                        print '!!! custom repair was performed : ', lino
                        line_buffer = re.sub('SKANDIA F\|RS\{KRINGS AB', 'SKANDIA FORSAKRINGS AB', line_buffer)
                        o.write(line_buffer + '\x0D\x0A')
                        d.write('REPAIRED LINE:' + cd + line_buffer + n)

                        cnt_bytes_written += len(line_buffer)
                        # correction for byte added/removed
                        #cnt_bytes_written -/+= 1
                        # report if read and written bytes differ
                        if cnt_bytes_read <> cnt_bytes_written: 
                            report_issue(lines_list, br=cnt_bytes_read, bw=cnt_bytes_written)
                            if one_by_one: sys.exit(2)

                    if lino in (18192, 18193):
                        '''
                        PROBLEM LINE NO: 18192|1|OLIMPICA COLOMBIA..
                        PROBLEM LINE NO: 18193|1|OLIMPICA COLOMBIA..
                        
                        'OLIMPICA COLOMBIA\|', 'OLIMPICA COLOMBIA'
                        '''
                        print '!!! custom repair was performed : ', lino
                        line_buffer = re.sub('OLIMPICA COLOMBIA\|', 'OLIMPICA COLOMBIA', line_buffer)
                        o.write(line_buffer + n)
                        d.write('REPAIRED LINE:' + cd + line_buffer + n)

                        cnt_bytes_written += len(line_buffer)
                        # correction for byte added/removed
                        #cnt_bytes_written -/+= 1
                        # correction for byte removed
                        cnt_bytes_written += 1
                        # report if read and written bytes differ
                        if cnt_bytes_read <> cnt_bytes_written: 
                            report_issue(lines_list, br=cnt_bytes_read, bw=cnt_bytes_written)
                            if one_by_one: sys.exit(2)

                    if lino in (21979, ):
                        '''
                        PROBLEM LINE NO: 21979|1|MINERAL|LHANDEL AG|..

                        1|MINERAL|LHANDEL AG|..
                        1|MINERAL\LHANDEL AG|..
                        1|MINERAL\LHANDEL AG|..
                        
                        'MINERAL\|LHANDEL AG', 'MINERAL\x5CLHANDEL AG'
                        '''
                        print '!!! custom repair was performed : ', lino
                        line_buffer = re.sub('MINERAL\|LHANDEL AG', 'MINERAL\x5CLHANDEL AG', line_buffer)
                        o.write(line_buffer + '\x0D\x0A')
                        d.write('REPAIRED LINE:' + cd + line_buffer + n)

                        cnt_bytes_written += len(line_buffer)
                        # correction for byte added/removed
                        #cnt_bytes_written -/+= 1
                        # report if read and written bytes differ
                        if cnt_bytes_read <> cnt_bytes_written:
                            report_issue(lines_list, br=cnt_bytes_read, bw=cnt_bytes_written)
                            if one_by_one: sys.exit(2)

                line_buffer = ''

            elif len(cols) < ref_cols_cnt:

                e.write(n)
                e.write("error item # %s" %(cnt_errors,) + n)
                cnt_errors += 1
                e.write("Raw data line number:\t%s" % (lino,) + n)
                e.write("Line original detail:\t%s" % (line,) + n)
                e.write("Lines accumulated: \t%s" % (line_buffer,) + n)
                e.write(n)
                
            else:
                print 'YOU SHOULD NEVER SEE THIS!'

        o.close()
        d.close()

        print n
        print filesmask
        print 'lines count of source data\t', lino
        print 'lines count cnt_equal_reference\t', cnt_equal_reference
        print 'lines count cnt_above_reference\t', cnt_above_reference
        print 'lines count cnt_below_reference\t', cnt_below_reference
        #print lino, cnt_equal_reference, cnt_joined_line
        print '== CHECKSUM 0 (expected 0)\t', cnt_joined_line + cnt_equal_reference + cnt_above_reference - lino
        print 'cnt_joined_line + cnt_equal_reference + cnt_above_reference - lino'

        print n
        print 'count of bytes read: \t', cnt_bytes_read
        print 'count of bytes written: \t', cnt_bytes_written
        print '== CHECKSUM 1 (expected 0)\t', cnt_bytes_read - cnt_bytes_written
        print 'cnt_bytes_read - cnt_bytes_written'

        e.write(n + n + n)
        e.write(filesmask + n + n)
        e.write('lines count of source data\t' + str(lino) + n)
        e.write('lines count cnt_equal_reference\t' + str(cnt_equal_reference) + n)
        e.write('lines count cnt_above_reference\t' + str(cnt_above_reference) + n)
        e.write('lines count cnt_below_reference\t' + str(cnt_below_reference) + n)
        e.write('== CHECKSUM 0 (expected 0)\t' + str(cnt_joined_line + cnt_equal_reference + cnt_above_reference - lino) + n)
        e.write('cnt_joined_line + cnt_equal_reference + cnt_above_reference - lino' + n)
        
        e.write(n)
        e.write('count of bytes read\t' + str(cnt_bytes_read) + n)
        e.write('count of bytes written\t' + str(cnt_bytes_written) + n)
        e.write('== CHECKSUM 1 (expected 0)\t' + str(cnt_bytes_read - cnt_bytes_written) + n)
        
        otxt = '''
                Log of lines with less than expected number of column delimiters:
                    

                Custom replacement of strings note:
                    None


                NOTE : Count of bytes read and count of bytes written does not equal actual files size
                because different end of lines of raw and output data are not counted into those counters.
                '''
        e.write(otxt + n)        
        e.close()

main()
